package vs1;

import Bots.Bots;
import Bots.Buffer;
import FALCON.FALCON;
import cz.cuni.amis.pogamut.base.agent.state.level0.IAgentState;
import cz.cuni.amis.pogamut.base.agent.state.level1.IAgentStateDown;
import cz.cuni.amis.pogamut.base.communication.worldview.event.IWorldEventListener;
import cz.cuni.amis.pogamut.base.communication.worldview.listener.annotation.EventListener;
import cz.cuni.amis.pogamut.base.communication.worldview.listener.annotation.ObjectClassEventListener;
import cz.cuni.amis.pogamut.base.utils.guice.AgentScoped;
import cz.cuni.amis.pogamut.base3d.worldview.object.event.WorldObjectAppearedEvent;
import cz.cuni.amis.pogamut.ut2004.agent.module.utils.TabooSet;
import cz.cuni.amis.pogamut.ut2004.agent.navigation.UT2004PathAutoFixer;
import cz.cuni.amis.pogamut.ut2004.bot.impl.UT2004Bot;
import cz.cuni.amis.pogamut.ut2004.bot.impl.UT2004BotModuleController;
import cz.cuni.amis.pogamut.ut2004.communication.messages.UT2004ItemType;
import cz.cuni.amis.pogamut.ut2004.communication.messages.gbcommands.Configuration;
import cz.cuni.amis.pogamut.ut2004.communication.messages.gbcommands.Initialize;
import cz.cuni.amis.pogamut.ut2004.communication.messages.gbcommands.RemoveRay;
import cz.cuni.amis.pogamut.ut2004.communication.messages.gbcommands.StopShooting;
import cz.cuni.amis.pogamut.ut2004.communication.messages.gbinfomessages.*;
import cz.cuni.amis.pogamut.ut2004.utils.UnrealUtils;
import cz.cuni.amis.utils.flag.FlagListener;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartFrame;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.StandardChartTheme;
import org.jfree.chart.plot.CategoryPlot;
import org.jfree.chart.plot.PlotOrientation;
import org.jfree.data.category.DefaultCategoryDataset;

import javax.vecmath.Vector3d;
import java.awt.*;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;


@AgentScoped
public abstract class TMP extends UT2004BotModuleController<UT2004Bot> {
    //主程序1
    protected static int agent_num = 1;                   // 智能体数量
    protected static int skill = 4;                     // 射击命中率
    protected int seed = 50;                      // 种子


    protected static Bots[] bots;                    // 智能体状态
    protected static FALCON[] agent;                // 网络模型
    protected int numAction = 4;                      // 动作数量
    public static int numState = 0;

    static {
        bots = new Bots[agent_num];
        agent = new FALCON[agent_num];
    }

    public TMP() {
        super();
        this.random = new Random(seed);
    }

    //逻辑{继承抽象类的都要重写逻辑}
    @Override
    public abstract void logic();

    ////////////////////////////////////////////////////////////////
    /////////////////////// 初始化   ///////////////////////////////
    ////////////////////////////////////////////////////////////////
    protected static boolean rw1 = false;              // 流程轨迹
    protected static boolean rw2 = false;              // 网络参数


    //main函数后面，监听智能体状态
    protected TabooSet<Item> tabooItems;            // 禁用物品集
    protected UT2004PathAutoFixer autoFixer;         // 自动检测是否会卡住,并删除坏边
    public int agt;
    static AtomicInteger count = new AtomicInteger(0);

    public TabooSet<Item> getTabooItems() {
        return tabooItems;
    }


    private FlagListener<IAgentState> botStateListener = changedValue -> {
        if (changedValue instanceof IAgentStateDown) {
            botAgentDown();
        }
    };

    @Override
    public void initializeController(UT2004Bot bot) {
        super.initializeController(bot);

        agt = count.get();
        count.incrementAndGet();

        getNavigation().setLogLevel(Level.OFF);
        getNavigation().getPathExecutor().getLog().setLevel(Level.OFF);
        getNavigation().getPathExecutor().removeAllStuckDetectors();
        getNavigation().getBackToNavGraph().clearStuckDetectors();
        getNavigation().getRunStraight().clearStuckDetectors();

        getNMNav().setLogLevel(Level.OFF);
        getNMNav().getBackToNavGraph().clearStuckDetectors();
        getNMNav().getRunStraight().clearStuckDetectors();
    }

    @Override
    public void prepareBot(UT2004Bot bot) {
        autoFixer = new UT2004PathAutoFixer(bot, getNavigation().getPathExecutor(), fwMap, aStar, navBuilder);
        tabooItems = new TabooSet<>(bot);     // 警用物品集合
        getWorldView().addEventListener(BotDamaged.class, botDamagedListener);       // 游戏环境里本机器人被攻击
        getWorldView().addEventListener(BotKilled.class, BotKilledListener);         // 游戏环境里本机器人被杀
        getWorldView().addEventListener(PlayerDamaged.class, PlayerDamagedListener);  // 游戏环境里
        getWorldView().addEventListener(PlayerKilled.class, PlayerKilledListener);   // 游戏环境里其他玩家被杀
        getWorldView().addEventListener(ItemPickedUp.class, ItemPickedUpListener);    // 捡起东西
        getWorldView().addEventListener(HearNoise.class, HearNoiseListener);          // 听到噪声


        getNavigation().getState().addListener(changedValue -> {
            switch (changedValue) {
                case PATH_COMPUTATION_FAILED:                                    // 如果路径计算错误
                case STUCK:                                                      // 如果路径堵塞
                case TARGET_REACHED:                                             // 到达目标
                    getNavigation().stopNavigation();                                 // 停止导航
                    break;
            }
        });

        weaponPrefs.addGeneralPref(UT2004ItemType.LIGHTNING_GUN, true);              // 添加武器偏好，使用武器的左键
        weaponPrefs.addGeneralPref(UT2004ItemType.SHOCK_RIFLE, true);
        weaponPrefs.addGeneralPref(UT2004ItemType.MINIGUN, false);
        weaponPrefs.addGeneralPref(UT2004ItemType.FLAK_CANNON, true);
        weaponPrefs.addGeneralPref(UT2004ItemType.ROCKET_LAUNCHER, true);
        weaponPrefs.addGeneralPref(UT2004ItemType.LINK_GUN, true);
        weaponPrefs.addGeneralPref(UT2004ItemType.ASSAULT_RIFLE, true);
        weaponPrefs.addGeneralPref(UT2004ItemType.BIO_RIFLE, true);
    }


    //prepareBot函数之后运行。
    @Override
    public Initialize getInitializeCommand() {
        reset(bots[agt]);
        return new Initialize().setName("AI").setTeam(1).setDesiredSkill(skill);
    }


    //4
    public static final String FRONT = "frontRay";                               // 定义一些字符串，用作感应射线 
    public static final String LEFT45 = "left45Ray";                             //
    public static final String LEFT90 = "left90Ray";                             //
    public static final String RIGHT45 = "right45Ray";                           //
    public static final String RIGHT90 = "right90Ray";                           //
    public static final String BEHIND = "behindRay";                             //
    public static final String BLEFT45 = "Bleft45Ray";                           //
    public static final String BRIGHT45 = "Bright45Ray";                         //

    //getInitializeCommand()函数之后运行，智能体初始化,一次性
    @Override
    public void botInitialized(GameInfo info, ConfigChange currentConfig, InitedMessage init) {  //重写智能体初始化   
        final int rayLength = (int) (UnrealUtils.CHARACTER_COLLISION_RADIUS * 2);  // 定义激光的长度 25 * 2
        boolean fastTrace = true;                                                  // 是否快速轨迹
        boolean floorCorrection = false;                                           // 地板正确性
        boolean traceActor = false;                                                // 跟踪器
        getAct().act(new RemoveRay("All"));                                        // 移除所有激光
        raycasting.createRay(LEFT45, new Vector3d(1, -1, 0), rayLength, fastTrace, floorCorrection, traceActor); // 创建左45度激光
        raycasting.createRay(FRONT, new Vector3d(1, 0, 0), rayLength, fastTrace, floorCorrection, traceActor);  //
        raycasting.createRay(RIGHT45, new Vector3d(1, 1, 0), rayLength, fastTrace, floorCorrection, traceActor);  // 
        raycasting.createRay(LEFT90, new Vector3d(0, -1, 0), rayLength, fastTrace, floorCorrection, traceActor); //
        raycasting.createRay(RIGHT90, new Vector3d(0, 1, 0), rayLength, fastTrace, floorCorrection, traceActor);  //  
        raycasting.createRay(BRIGHT45, new Vector3d(1, -1, 0), -rayLength, fastTrace, floorCorrection, traceActor);//
        raycasting.createRay(BEHIND, new Vector3d(1, 0, 0), -rayLength, fastTrace, floorCorrection, traceActor); //
        raycasting.createRay(BLEFT45, new Vector3d(1, 1, 0), -rayLength, fastTrace, floorCorrection, traceActor);  //  
        raycasting.endRayInitSequence();                                            // 开始监听激光
        getAct().act(new Configuration().setDrawTraceLines(true).setAutoTrace(true)); // 是否显示激光
    }


    /////////////////////////////////////////////////////////////////
    /////////////////////// 一堆环境监控   /////////////////////////
    ////////////////////////////////////////////////////////////////
    //所有事件监听
    //被打
    public IWorldEventListener<BotDamaged> botDamagedListener = event -> {
    };

    //打人
    public IWorldEventListener<PlayerDamaged> PlayerDamagedListener = new IWorldEventListener<PlayerDamaged>() {
        @Override
        public void notify(PlayerDamaged event) {
            bots[agt].r = 0.5;
            //如果击中敌人就造成0.75奖励
        }
    };

    boolean dead_help = false;  //各线程单独使用
    //被杀
    public IWorldEventListener<BotKilled> BotKilledListener = new IWorldEventListener<BotKilled>() {
        @Override   //重写
        public void notify(BotKilled event) {
            getNavigation().stopNavigation();          // 停止导航
            getAct().act(new StopShooting());     // 停止射击
            if (!dead_help) {                // 没啥用
                bots[agt].r = 0;                      // 设定奖励值为0
                bots[agt].end = true;                 // 终止
                fi = true;
                bots[agt].die++;                      // 死亡数增加
                Learning(agent[agt], bots[agt]);
                reset(bots[agt]);                     // 重置机器人状态
                bots[agt].fitness2 += 1;
                bots[agt].KD = (double) bots[agt].fitness1 / (double) bots[agt].fitness2;
			/*
				if(bots[agt].kill==10 || bots[agt].die==10) {
					EVAL(agent,bots);             // 评估打印各项参数
					bots[agt].kill=0;             // 将击杀数设为0
					bots[agt].die=0;              // 将死亡数设为0
					agent[agt].QEpsilon-=agent[agt].QEpsilonDecay; //
					System.out.println(agent[agt].QEpsilon);
				}
			*/
            }

            if (rw1) {
                try {
                    agent[agt].writefile(agt, "agent");  // 记录智能体网络参数
                } catch (Exception e) {
                    System.out.println("error");
                }
            }
            if (rw2) {
                try {
                    bots[agt].writefile(agt, "bots");    // 记录智能体动作轨迹
                } catch (Exception e) {
                    System.out.println("error");
                }
            }
        }
    };


    //杀敌
    public IWorldEventListener<PlayerKilled> PlayerKilledListener = new IWorldEventListener<PlayerKilled>() {
        @Override
        public void notify(PlayerKilled event) {
            if (event.getKiller() == info.getId()) {                  // 如果击杀者为智能体
                getNavigation().stopNavigation();                         // 停止导航
                getAct().act(new StopShooting());                    // 停止射击
                bots[agt].r = 1.0;                                     // 奖励值为1
                bots[agt].kill += 1;                                   // 击杀数+1
                //bots[agt].end = true;                                // 任务结束
                //Learning(agent[agt],bots[agt]);                      // 学习这段记忆
                bots[agt].enemy = null;                              // 没有敌人了
                bots[agt].fitness1 += 1;
                if (bots[agt].fitness2 != 0)
                    bots[agt].KD = (double) bots[agt].fitness1 / bots[agt].fitness2;
                else
                    bots[agt].KD = bots[agt].fitness1;
				/*
				if(bots[agt].kill==10 || bots[agt].die==10) {
					EVAL(agent,bots);             // 评估打印各项参数
					bots[agt].kill=0;             // 将击杀数设为0
					bots[agt].die=0;              // 将死亡数设为0
					agent[agt].QEpsilon-=agent[agt].QEpsilonDecay; //
					System.out.println(agent[agt].QEpsilon);
				}
			*/
                //dead_help=true;
                //getAct().act(new Respawn());
            }


        }
    };


    //拾取物品
    public IWorldEventListener<ItemPickedUp> ItemPickedUpListener = event -> {
//			if(bots[agt].item!=null && event.getType() == bots[agt].item.getType()) {  //如果智能体正在导向目标且拾取到的是目标
//				//bots[agt].r=0.5;  //奖励值设为0.75
//			}
    };

    //监听噪音
    public IWorldEventListener<HearNoise> HearNoiseListener = event -> { // 如果发出噪音
    };

    //碰面
    @ObjectClassEventListener(eventClass = WorldObjectAppearedEvent.class, objectClass = Player.class)
    public void playerAppeared(WorldObjectAppearedEvent<Player> event) {      //如果智能体相遇
    }

    @EventListener(eventClass = TeamChat.class)
    public void teamChat(TeamChat event) { //智能体之间的对话
    }

    /////////////////////////////////////////////////////////////////
    ///////////////////////奖励函数   ///////////////////////////////////
    ////////////////////////////////////////////////////////////////
    public abstract void reward_function(Bots bots);

    ////////////////////////////////////////////////////////////////
    /////////////////////// 状态   ////////////////////////////////////
    ///////////////////////////////////////////////////////////////


    public abstract void observe(Bots bots);

    public abstract void SelectVAct(Bots bots);

    //选择动作
    public abstract void DoAct(Bots bots, int action);


    public int step = 0;
//    static int WriteInTarget=1;
//    int KK=128;

    int drawi1 = 0;

    public void EVAL(FALCON[] agent, Bots[] bots) {
        double rewards;
        int die;
        int kill;
        if (agt == 0) {
//    		int c=agt;
            System.out.println("KD");
            kill = 0;
            die = 0;
            for (int v = 0; v < agent_num; v++) {
                kill += bots[v].kill;
                die += bots[v].die;
                bots[v].die = 0;
                bots[v].kill = 0;
            }
            if (die != 0) System.out.println(((double) kill / (double) die));
            else System.out.println((double) kill);
            drawLine((double) kill / die, drawi1);


            rewards = 0;
            System.out.println("bots.rewards");
            for (int v = 0; v < agent_num; v++) {
                rewards += bots[v].rewards;
                bots[v].rewards = 0.0;
            }
            System.out.println(rewards);


            int numCode = 0;
            System.out.println("numCode");
            for (int v = 0; v < agent_num; v++) {
                numCode += agent[v].numCode;
            }
            System.out.println(numCode);
            System.out.println("step");
            int step = 0;
            for (int v = 0; v < agent_num; v++) {
                step += bots[v].step;
                bots[v].step = 0;
            }
            System.out.println(step);
//    		step=0;

            int TF_num = 0;
            System.out.println("TF_num");
            for (int v = 0; v < agent_num; v++) {
                TF_num += bots[v].TF_num;
                bots[v].TF_num = 0;
            }
            System.out.println(TF_num);
    		
    		/*
    		System.out.println("action");
    		int action0=0;
    		int action1=0;
    		int action2=0;
    		int action3=0;
    		   
    		for(int v=0;v<agent_num;v++) {
    			action0+=bots[v].action0;
    			bots[v].action0=0;
    			action1+=bots[v].action1;
    			bots[v].action1=0;
    			action2+=bots[v].action2;
    			bots[v].action2=0;
    			action3+=bots[v].action3;
    			bots[v].action3=0;
    		}
    		
    		int action=action0+action1+action2+action3;
    		System.out.println((double)action0/(double)action);
    		System.out.println((double)action1/(double)action);
    		System.out.println((double)action2/(double)action);
    		System.out.println((double)action3/(double)action);

    		System.out.println("a");
    		int a0=0;
    		int a1=0;
    		int a2=0;
    		int a3=0;
    		for(int v=0;v<agent_num;v++) {
    			a0+=bots[v].a0;
    			bots[v].a0=0;
    			a1+=bots[v].a1;
    			bots[v].a1=0;
    			a2+=bots[v].a2;
    			bots[v].a2=0;
    			a3+=bots[v].a3;
    			bots[v].a3=0;
    		}
    		int a=a0+a1+a2+a3;
    		if(a!=0) {
    			System.out.println((double)a0/(double)a);
    			System.out.println((double)a1/(double)a);
    			System.out.println((double)a2/(double)a);
    			System.out.println((double)a3/(double)a);
    		}else {
    			System.out.println((double)a0);
    			System.out.println((double)a1);
    			System.out.println((double)a2);
    			System.out.println((double)a3);
    		}
    		*/
//    		System.out.println("maxQ");
//    	    int maxQ0=0;
//    	    int maxQ1=0;
//    	    int maxQ2=0;
//    	    int maxQ3=0;
//    	    int maxQ4=0;
//    	    int maxQ5=0;
//    	    int maxQ6=0;
//    	    int maxQ7=0;
//    	    int maxQ8=0;
//    	    int maxQ9=0;
//    		for(int v=0;v<agent_num;v++) {
//    			maxQ0+=agent[v].maxQ0;
//    			agent[v].maxQ0=0;
//    			maxQ1+=agent[v].maxQ1;
//    			agent[v].maxQ1=0;
//    			maxQ2+=agent[v].maxQ2;
//    			agent[v].maxQ2=0;
//    			maxQ3+=agent[v].maxQ3;
//    			agent[v].maxQ3=0;
//    			maxQ4+=agent[v].maxQ4;
//    			agent[v].maxQ4=0;
//    			maxQ5+=agent[v].maxQ5;
//    			agent[v].maxQ5=0;
//    			maxQ6+=agent[v].maxQ6;
//    			agent[v].maxQ6=0;
//    			maxQ7+=agent[v].maxQ7;
//    			agent[v].maxQ7=0;
//    			maxQ8+=agent[v].maxQ8;
//    			agent[v].maxQ8=0;
//    			maxQ9+=agent[v].maxQ9;
//    			agent[v].maxQ9=0;
//    		}
//    		int maxQ=maxQ0+maxQ1+maxQ2+maxQ3+maxQ4+maxQ5+maxQ6+maxQ7+maxQ8+maxQ9;
//    		    System.out.println((double)maxQ0/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ1/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ2/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ3/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ4/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ5/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ6/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ7/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ8/(maxQ+ 1e-6));
//    			System.out.println((double)maxQ9/(maxQ+ 1e-6));
//
//        		System.out.println("minQ");
//        	    int minQ0=0;
//        	    int minQ1=0;
//        	    int minQ2=0;
//        	    int minQ3=0;
//        	    int minQ4=0;
//        	    int minQ5=0;
//        	    int minQ6=0;
//        	    int minQ7=0;
//        	    int minQ8=0;
//        	    int minQ9=0;
//        		for(int v=0;v<agent_num;v++) {
//        			minQ0+=agent[v].minQ0;
//        			agent[v].minQ0=0;
//        			minQ1+=agent[v].minQ1;
//        			agent[v].minQ1=0;
//        			minQ2+=agent[v].minQ2;
//        			agent[v].minQ2=0;
//        			minQ3+=agent[v].minQ3;
//        			agent[v].minQ3=0;
//        			minQ4+=agent[v].minQ4;
//        			agent[v].minQ4=0;
//        			minQ5+=agent[v].minQ5;
//        			agent[v].minQ5=0;
//        			minQ6+=agent[v].minQ6;
//        			agent[v].minQ6=0;
//        			minQ7+=agent[v].minQ7;
//        			agent[v].minQ7=0;
//        			minQ8+=agent[v].minQ8;
//        			agent[v].minQ8=0;
//        			minQ9+=agent[v].minQ9;
//        			agent[v].minQ9=0;
//        		}
//        		int minQ=minQ0+minQ1+minQ2+minQ3+minQ4+minQ5+minQ6+minQ7+minQ8+minQ9;
//        		    System.out.println((double)minQ0/(minQ+ 1e-6));
//        			System.out.println((double)minQ1/(minQ+ 1e-6));
//        			System.out.println((double)minQ2/(minQ+ 1e-6));
//        			System.out.println((double)minQ3/(minQ+ 1e-6));
//        			System.out.println((double)minQ4/(minQ+ 1e-6));
//        			System.out.println((double)minQ5/(minQ+ 1e-6));
//        			System.out.println((double)minQ6/(minQ+ 1e-6));
//        			System.out.println((double)minQ7/(minQ+ 1e-6));
//        			System.out.println((double)minQ8/(minQ+ 1e-6));
//        			System.out.println((double)minQ9/(minQ+ 1e-6));
//
//    		System.out.println("mmQ");
//    	    int mmQ0=0;
//    	    int mmQ1=0;
//    	    int mmQ2=0;
//    	    int mmQ3=0;
//    	    int mmQ4=0;
//    	    int mmQ5=0;
//    	    int mmQ6=0;
//    	    int mmQ7=0;
//    	    int mmQ8=0;
//    	    int mmQ9=0;
//    		for(int v=0;v<agent_num;v++) {
//    			mmQ0+=agent[v].mmQ0;
//    			agent[v].mmQ0=0;
//    			mmQ1+=agent[v].mmQ1;
//    			agent[v].mmQ1=0;
//    			mmQ2+=agent[v].mmQ2;
//    			agent[v].mmQ2=0;
//    			mmQ3+=agent[v].mmQ3;
//    			agent[v].mmQ3=0;
//    			mmQ4+=agent[v].mmQ4;
//    			agent[v].mmQ4=0;
//    			mmQ5+=agent[v].mmQ5;
//    			agent[v].mmQ5=0;
//    			mmQ6+=agent[v].mmQ6;
//    			agent[v].mmQ6=0;
//    			mmQ7+=agent[v].mmQ7;
//    			agent[v].mmQ7=0;
//    			mmQ8+=agent[v].mmQ8;
//    			agent[v].mmQ8=0;
//    			mmQ9+=agent[v].mmQ9;
//    			agent[v].mmQ9=0;
//    		}
//    		int mmQ=mmQ0+mmQ1+mmQ2+mmQ3+mmQ4+mmQ5+mmQ6+mmQ7+mmQ8+mmQ9;
//    		    System.out.println((double)mmQ0/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ1/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ2/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ3/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ4/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ5/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ6/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ7/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ8/(mmQ+ 1e-6));
//    			System.out.println((double)mmQ9/(mmQ+ 1e-6));
            drawi1++;
        }
    }


    public void BUFFER(Bots bots) {
        Buffer b = new Buffer(bots.numState, bots.numAction);    // 定义新的Buffer片
        b.Replay_Buffer(bots.CurrentStat, bots.action, bots.r, bots.NextStat, bots.end); // 存储当前数据
        bots.buff.add(b);                                                      // 存取buffer数量+1
        //bots.prior_action=bots.action;                                    
    }


    public boolean fi = true;

    public void reset(Bots bots) {
        bots.reset();    //将所有变量都设为默认值
        bots.enemy = null; //将敌人设定为NULL
        getNavigation().stopNavigation(); //停止导航
        getAct().act(new StopShooting()); //停止射击
        if (!fi) {              // 最开始reset无法获得权限
            bots.action = -1;    // 不需要动作
            observe(bots);     //
        }
        fi = false;
        bots.CurrentStat = Arrays.copyOf(bots.stat, bots.stat.length);
        bots.NextStat = Arrays.copyOf(bots.stat, bots.stat.length);
    }

    public void Learning(FALCON agent, Bots bots) {
        observe(bots);   // 观测状态
        bots.NextStat = Arrays.copyOf(bots.stat, bots.stat.length);
        reward_function(bots); // 观测奖励函数
        bots.rewards += bots.r;  // 累计奖励
        BUFFER(bots);    // 存储step
        agent.Q_Function_update(bots.buff.get(bots.buff.size() - 1)); //更新网络参数
        bots.r = 0;        // 将奖励设为0
        if (bots.end) bots.end = false; // 设定不结束
    }

    public void perf(FALCON agent, Bots bots) {
        observe(bots);                                                //观测
        bots.CurrentStat = Arrays.copyOf(bots.stat, bots.stat.length); //现在的状态
        int selectedAction;
        SelectVAct(bots);                                         // 判断可执行动作
//选择可执行动作
        if (random.nextDouble() < FALCON.QEpsilon) {                     // 随机行为
            do {
                selectedAction = (int) (random.nextDouble() * numAction);
            } while (!bots.vact[selectedAction]);
            bots.action = selectedAction;                              //
        } else {
            bots.action = agent.sPolicy(bots.CurrentStat, bots.vact); // 智能体选择Q值
        }
        bots.r = 0;                                                    // 奖励值设为0
        bots.lastheal = info.getHealth() / 100.0;              // 最后血量
        bots.hitdamage = 0;                                            // 伤害值
        DoAct(bots, bots.action);                                     // 执行动作
    }

    // java 表格
    static CategoryPlot mPlot;
    static JFreeChart mChart;
    static DefaultCategoryDataset mDataset = new DefaultCategoryDataset();
    static StandardChartTheme mChartTheme;
    static ChartFrame mChartFrame;

    public static void drawLine(double value, int i) {
        if (mChartFrame != null) mChartFrame.setVisible(false);

        mChartTheme = new StandardChartTheme("CN");
        mChartTheme.setLargeFont(new Font("黑体", Font.BOLD, 20));
        mChartTheme.setExtraLargeFont(new Font("宋体", Font.PLAIN, 15));
        mChartTheme.setRegularFont(new Font("宋体", Font.PLAIN, 15));
        ChartFactory.setChartTheme(mChartTheme);

        mDataset.addValue(value, "improve", String.valueOf(i));

        mChart = ChartFactory.createLineChart(
                "折线图",//图名字
                "trait",//横坐标
                "value",//纵坐标
                mDataset,//数据集
                PlotOrientation.VERTICAL,
                true, // 显示图例
                true, // 采用标准生成器
                false);// 是否生成超链接

        mPlot = (CategoryPlot) mChart.getPlot();
        mPlot.setBackgroundPaint(Color.LIGHT_GRAY);
        mPlot.setRangeGridlinePaint(Color.BLUE);//背景底部横虚线
        mPlot.setOutlinePaint(Color.RED);//边界线

        mChartFrame = new ChartFrame("折线图", mChart);
        mChartFrame.pack();
        mChartFrame.setVisible(true);
    }

}